{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(Z):\n",
    "    \"\"\"\n",
    "    Implements the sigmoid activation in numpy\n",
    "    \n",
    "    Arguments:\n",
    "    Z -- numpy array of any shape\n",
    "    \n",
    "    Returns:\n",
    "    A -- output of sigmoid(z), same shape as Z\n",
    "    cache -- returns Z as well, useful during backpropagation\n",
    "    \"\"\"\n",
    "    \n",
    "    A = 1/(1+np.exp(-Z))\n",
    "    cache = Z\n",
    "    \n",
    "    return A, cache\n",
    "\n",
    "def relu(Z):\n",
    "    \"\"\"\n",
    "    Implement the RELU function.\n",
    "\n",
    "    Arguments:\n",
    "    Z -- Output of the linear layer, of any shape\n",
    "\n",
    "    Returns:\n",
    "    A -- Post-activation parameter, of the same shape as Z\n",
    "    cache -- a python dictionary containing \"A\" ; stored for computing the backward pass efficiently\n",
    "    \"\"\"\n",
    "    \n",
    "    A = np.maximum(0,Z)\n",
    "    \n",
    "    assert(A.shape == Z.shape)\n",
    "    \n",
    "    cache = Z \n",
    "    return A, cache\n",
    "\n",
    "def relu_backward(dA, cache):\n",
    "    \"\"\"\n",
    "    Implement the backward propagation for a single RELU unit.\n",
    "\n",
    "    Arguments:\n",
    "    dA -- post-activation gradient, of any shape\n",
    "    cache -- 'Z' where we store for computing backward propagation efficiently\n",
    "\n",
    "    Returns:\n",
    "    dZ -- Gradient of the cost with respect to Z\n",
    "    \"\"\"\n",
    "    \n",
    "    Z = cache\n",
    "    dZ = np.array(dA, copy=True) # just converting dz to a correct object.\n",
    "    \n",
    "    # When z <= 0, you should set dz to 0 as well. \n",
    "    dZ[Z <= 0] = 0\n",
    "    \n",
    "    assert (dZ.shape == Z.shape)\n",
    "    \n",
    "    return dZ\n",
    "\n",
    "def sigmoid_backward(dA, cache):\n",
    "    \"\"\"\n",
    "    Implement the backward propagation for a single SIGMOID unit.\n",
    "\n",
    "    Arguments:\n",
    "    dA -- post-activation gradient, of any shape\n",
    "    cache -- 'Z' where we store for computing backward propagation efficiently\n",
    "\n",
    "    Returns:\n",
    "    dZ -- Gradient of the cost with respect to Z\n",
    "    \"\"\"\n",
    "    \n",
    "    Z = cache\n",
    "    \n",
    "    s = 1/(1+np.exp(-Z))\n",
    "    dZ = dA * s * (1-s)\n",
    "    \n",
    "    assert (dZ.shape == Z.shape)\n",
    "    \n",
    "    return dZ\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_parameters(n_x,n_h,n_y):\n",
    "    \"\"\"\n",
    "    Argument:\n",
    "    n_x -- size of the input layer\n",
    "    n_h -- size of the hidden layer\n",
    "    n_y -- size of the output layer\n",
    "    Returns:\n",
    "    parameters -- python dictionary containing your parameters:\n",
    "    W1 -- weight matrix of shape (n_h, n_x)\n",
    "    b1 -- bias vector of shape (n_h, 1)\n",
    "    W2 -- weight matrix of shape (n_y, n_h)\n",
    "    b2 -- bias vector of shape (n_y, 1)\n",
    "    \"\"\"\n",
    "    W1=np.random.randn(n_h,n_x)*0.01\n",
    "    b1=np.zeros(n_h,1)\n",
    "    W2=np.random.randn(n_y,n_x)*0.01\n",
    "    b2=np.zeros(n_y,1)\n",
    "    assert (W1.shape==(n_h,n_x))\n",
    "    assert (b1.shape==(n_h,1))\n",
    "    assert (W2.shape==(n_y,n_h))\n",
    "    assert (b2.shape==(n_y,1))\n",
    "    \n",
    "    parameters={\n",
    "        \"W1\":W1,\n",
    "        \"b1\":b1,\n",
    "        \"W2\":W2,\n",
    "        \"b2\":b2\n",
    "    }\n",
    "    return parameters\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_parameters_deep(layer_dims):\n",
    "    \"\"\"\n",
    "    Arguments:\n",
    "    layer_dims -- python array (list) containing the dimensions of each\n",
    "    ,! layer in our network\n",
    "    Returns:\n",
    "    parameters -- python dictionary containing your parameters \"W1\",\n",
    "    ,! \"b1\", ..., \"WL\", \"bL\":\n",
    "    Wl -- weight matrix of shape (layer_dims[l], layer_dims[l-1])\n",
    "    bl -- bias vector of shape (layer_dims[l], 1)\n",
    "    \"\"\"\n",
    "    np.random.seed(3)\n",
    "    parameters = {}\n",
    "    L = len(layer_dims) # number of layers in the network\n",
    "    for l in range(1, L):\n",
    "        ### START CODE HERE ### ( 2 lines of code)\n",
    "        parameters['W' + str(l)] = np.random.randn(layer_dims[l],\n",
    "        layer_dims[l-1]) * 0.01\n",
    "        parameters['b' + str(l)] = np.zeros((layer_dims[l], 1))\n",
    "        ### END CODE HERE ###\n",
    "        assert(parameters['W' + str(l)].shape == (layer_dims[l],\n",
    "         layer_dims[l-1]))\n",
    "        assert(parameters['b' + str(l)].shape == (layer_dims[l], 1))\n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_forwar(A,W,b):\n",
    "    \"\"\"\n",
    "    Implement the linear part of a layer's forward propagation.\n",
    "    Arguments:\n",
    "    A -- activations from previous layer (or input data): (size of\n",
    "    ,! previous layer, number of examples)\n",
    "    W -- weights matrix: numpy array of shape (size of current layer,\n",
    "    ,! size of previous layer)\n",
    "    b -- bias vector, numpy array of shape (size of the current layer,\n",
    "    ,! 1)\n",
    "    Returns:\n",
    "    Z -- the input of the activation function, also called\n",
    "    ,! pre-activation parameter\n",
    "    cache -- a python dictionary containing \"A\", \"W\" and \"b\" ; stored\n",
    "    ,! for computing the backward pass efficiently\n",
    "    \"\"\"\n",
    "    Z=W @ A +b\n",
    "    assert (Z.shape==(W.shape[0],A.shape[1]))\n",
    "    cache=(A,W,b)\n",
    "    return Z,cache\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(self, z):\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def linear_activation_forward(A_prev, W, b, activation):\n",
    "    \"\"\"\n",
    "    Implement the forward propagation for the LINEAR->ACTIVATION layer\n",
    "    Arguments:\n",
    "    A_prev -- activations from previous layer (or input data): (size of\n",
    "    ,! previous layer, number of examples)\n",
    "    W -- weights matrix: numpy array of shape (size of current layer,\n",
    "    ,! size of previous layer)\n",
    "    b -- bias vector, numpy array of shape (size of the current layer,\n",
    "    ,! 1)\n",
    "    activation -- the activation to be used in this layer, stored as a\n",
    "    ,! text string: \"sigmoid\" or \"relu\"\n",
    "    Returns:\n",
    "    A -- the output of the activation function, also called the\n",
    "    ,! post-activation value\n",
    "    cache -- a python dictionary containing \"linear_cache\" and\n",
    "    ,! \"activation_cache\";\n",
    "    stored for computing the backward pass efficiently\n",
    "    \"\"\"\n",
    "    if activation==\"sigmoid\":\n",
    "        Z.linear_cache=linear_forwar(A_prev,W,b)\n",
    "        A,activation_cache=sigmoid(Z)\n",
    "    elif activation==\"relu\":\n",
    "        Z,linear_cache=linear_activation_forward(A_prev,W,b)\n",
    "        A,activation_cache=relu(Z)\n",
    "    \n",
    "    assert (A.shape==(W.shape[0],A_prev.shape[1]))\n",
    "    cache=(linear_cache,activation_cache)\n",
    "    return A.cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def L_model_forward(X, parameters):\n",
    "    \"\"\"\n",
    "    Implement forward propagation for the\n",
    "    ,! [LINEAR->RELU]*(L-1)->LINEAR->SIGMOID computation\n",
    "    Arguments:\n",
    "    X -- data, numpy array of shape (input size, number of examples)\n",
    "    parameters -- output of initialize_parameters_deep()\n",
    "    Returns:\n",
    "    AL -- last post-activation value\n",
    "    caches -- list of caches containing:\n",
    "    every cache of linear_relu_forward() (there are L-1 of\n",
    "    ,! them, indexed from 0 to L-2)\n",
    "    the cache of linear_sigmoid_forward() (there is one,\n",
    "    ,! indexed L-1)\n",
    "    \"\"\"\n",
    "    caches=[]\n",
    "    A=X\n",
    "    L=len(parameters)\n",
    "    for l in range(1,L):\n",
    "        A_prev=A\n",
    "        A,cache=linear_activation_forward(A_prev,parameters['W'+str(l)],\n",
    "                                         parameters['b'+str(l)],activation=\"relu\")\n",
    "        caches.append(cache)\n",
    "        \n",
    "    \n",
    "    AL,cacahe=linear_activation_forward(A,parameters['W'+str(L)],\n",
    "                                        parameters['b'+str(L)],activation=\"sigmoid\")\n",
    "    caches.append(cache)\n",
    "    assert(AL.shape==(1,X.shape[1]))\n",
    "    return AL.cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_cost(AL, Y):\n",
    "    \"\"\"\n",
    "    Implement the cost function defined by equation (7).\n",
    "    Arguments:\n",
    "    AL -- probability vector corresponding to your label predictions,\n",
    "    ,! shape (1, number of examples)\n",
    "    Y -- true \"label\" vector (for example: containing 0 if non-cat, 1\n",
    "    ,! if cat), shape (1, number of examples)\n",
    "    Returns:\n",
    "    cost -- cross-entropy cost\n",
    "    \"\"\"\n",
    "    m=Y.shape[1]\n",
    "    cost=-(Y @ np.log(AL.T) +(1-Y) @ np.log(1-AL).T)/m\n",
    "    cost=np.squeeze(cost) #reduce a dim\n",
    "    assert cost,shape==()\n",
    "    return cost\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_backward(dZ, cache):\n",
    "    \"\"\"\n",
    "    Implement the linear portion of backward propagation for a single\n",
    "    ,! layer (layer l)\n",
    "    Arguments:\n",
    "    dZ -- Gradient of the cost with respect to the linear output (of\n",
    "    ,! current layer l)\n",
    "    cache -- tuple of values (A_prev, W, b) coming from the forward\n",
    "    ,! propagation in the current layer\n",
    "    Returns:\n",
    "    dA_prev -- Gradient of the cost with respect to the activation (of\n",
    "    ,! the previous layer l-1), same shape as A_prev\n",
    "    dW -- Gradient of the cost with respect to W (current layer l),\n",
    "    ,! same shape as W\n",
    "    db -- Gradient of the cost with respect to b (current layer l),\n",
    "    ,! same shape as b\n",
    "    \"\"\"\n",
    "    A_prev,W,b=cache\n",
    "    m=A_prev.shape[1]\n",
    "    \n",
    "    dW=(dZ @ A_prev.T)/m\n",
    "    db=np.sum(dZ,axis=1,keepdims=True)/m\n",
    "    dA_prev=W.T @ dZ\n",
    "    assert (dA_prev.shape == A_prev.shape)\n",
    "    assert (dW.shape == W.shape)\n",
    "    assert (db.shape == b.shape)\n",
    "    return dA_prev, dW, db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu_backward(dA,activation_cache):\n",
    "    \n",
    "\n",
    "def linear_activation_backward(dA,cache,activation):\n",
    "    \"\"\"\n",
    "    Implement the backward propagation for the LINEAR->ACTIVATION\n",
    "    ,! layer.\n",
    "    Arguments:\n",
    "    dA -- post-activation gradient for current layer l\n",
    "    cache -- tuple of values (linear_cache, activation_cache) we store\n",
    "    ,! for computing backward propagation efficiently\n",
    "    activation -- the activation to be used in this layer, stored as a\n",
    "    ,! text string: \"sigmoid\" or \"relu\"\n",
    "    Returns:\n",
    "    dA_prev -- Gradient of the cost with respect to the activation (of\n",
    "    ,! the previous layer l-1), same shape as A_prev\n",
    "    dW -- Gradient of the cost with respect to W (current layer l),\n",
    "    ,! same shape as W\n",
    "    db -- Gradient of the cost with respect to b (current layer l),\n",
    "    ,! same shape as b\n",
    "    \"\"\"\n",
    "    \n",
    "    linear_cacahe,activation_cache=cache\n",
    "    \n",
    "    if activation==\"relu\":\n",
    "        dZ=relu_backward(dA,activation_cache)\n",
    "        dA_prev,dW,db=linear_backward(dZ,linear_cacahe)\n",
    "    \n",
    "    elif activation==\"sigmoid\":\n",
    "        dZ=sigmoid_backward(dA,activation_cache)\n",
    "        dA_prev,dW,db=linear_backward(dZ,linear_cacahe)\n",
    "    \n",
    "    return dA_prev,dW,db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def L_model_backward(AL,Y,caches):\n",
    "    \"\"\"\n",
    "    Implement the backward propagation for the [LINEAR->RELU] * (L-1)\n",
    "    ,! -> LINEAR -> SIGMOID group\n",
    "        Arguments:\n",
    "    AL -- probability vector, output of the forward propagation\n",
    "    ,! (L_model_forward())\n",
    "    Y -- true \"label\" vector (containing 0 if non-cat, 1 if cat)\n",
    "    caches -- list of caches containing:\n",
    "    every cache of linear_activation_forward() with \"relu\"\n",
    "    ,! (it's caches[l], for l in range(L-1) i.e l = 0...L-2)\n",
    "    the cache of linear_activation_forward() with \"sigmoid\"\n",
    "    ,! (it's caches[L-1])\n",
    "    Returns:\n",
    "    grads -- A dictionary with the gradients\n",
    "    grads[\"dA\" + str(l)] = ...\n",
    "    grads[\"dW\" + str(l)] = ...\n",
    "    grads[\"db\" + str(l)] = ...\n",
    "    \"\"\"\n",
    "    \n",
    "    grads={}\n",
    "    L=len(caches)\n",
    "    m=AL.shape[1]\n",
    "    Y=Y.reshape(AL.shape)\n",
    "    \n",
    "    dAL= - (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL))\n",
    "    \n",
    "    current_cache = caches[L-1]\n",
    "    grads[\"dA\" + str(L)], grads[\"dW\" + str(L)], grads[\"db\" + str(L)] =linear_activation_backward(dAL, current_cache, \"sigmoid\")\n",
    "    \n",
    "    for l in reversed(range(L-1)):\n",
    "        current_cache=caches[l]\n",
    "        dA_prev_temp,dW_temp,db_temp=linear_activation_backward(grads[\"dA\"+str(l+2)],current_cache,\"relu\")\n",
    "        grads[\"dA\"+str(l+1)]=dA_prev_temp\n",
    "        grads[\"dW\"+str(l+1)]=dW_temp\n",
    "        grads[\"db\"+str(l+1)]=db_temp\n",
    "        return grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_parameters(parameters, grads, learning_rate):\n",
    "    \"\"\"\n",
    "    Update parameters using gradient descent\n",
    "    Arguments:\n",
    "    parameters -- python dictionary containing your parameters\n",
    "    grads -- python dictionary containing your gradients, output of\n",
    "    ,! L_model_backward\n",
    "    Returns:\n",
    "    parameters -- python dictionary containing your updated parameters\n",
    "    parameters[\"W\" + str(l)] = ...\n",
    "    parameters[\"b\" + str(l)] = ...\n",
    "    \"\"\"\n",
    "    L = len(parameters) // 2 # number of layers in the neural network\n",
    "    # Update rule for each parameter. Use a for loop.\n",
    "    for l in range(L):\n",
    "        parameters[\"W\" + str(l+1)] = parameters[\"W\" + str(l+1)] -learning_rate * grads[\"dW\" + str(l + 1)]\n",
    "        parameters[\"b\" + str(l+1)] = parameters[\"b\" + str(l+1)] -learning_rate * grads[\"db\" + str(l + 1)]\n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
